#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
数据验证工具
用于验证数据的完整性和有效性
"""

import os
import json
import logging
import pandas as pd
import numpy as np
from datetime import datetime

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

class DataValidator:
    """数据验证器，用于检查数据的完整性和有效性"""
    
    def __init__(self, config_path='config/config.yaml'):
        """
        初始化数据验证器
        
        Args:
            config_path: 配置文件路径
        """
        self.config_path = config_path
        self.validation_results = {}
        
    def validate_stock_basic_data(self, stock_data):
        """
        验证股票基础数据的完整性和有效性
        
        Args:
            stock_data: 股票基础数据字典
            
        Returns:
            dict: 验证结果
        """
        if not stock_data:
            return {
                'valid': False,
                'missing_fields': ['all'],
                'invalid_fields': [],
                'message': '股票数据为空'
            }
            
        # 必要字段列表
        required_fields = ['code', 'name', 'industry', 'pe_ttm', 'pb', 'total_mv', 'income_yoy']
        
        # 检查缺失字段
        missing_fields = [field for field in required_fields if field not in stock_data or stock_data[field] is None]
        
        # 检查无效字段
        invalid_fields = []
        
        # 检查pe_ttm是否为有效值
        if 'pe_ttm' in stock_data and stock_data['pe_ttm'] is not None:
            try:
                pe_ttm = float(stock_data['pe_ttm'])
                if pe_ttm < 0 or pe_ttm > 1000:  # 设置合理范围
                    invalid_fields.append('pe_ttm')
            except (ValueError, TypeError):
                invalid_fields.append('pe_ttm')
                
        # 检查pb是否为有效值
        if 'pb' in stock_data and stock_data['pb'] is not None:
            try:
                pb = float(stock_data['pb'])
                if pb < 0 or pb > 100:  # 设置合理范围
                    invalid_fields.append('pb')
            except (ValueError, TypeError):
                invalid_fields.append('pb')
                
        # 检查total_mv是否为有效值
        if 'total_mv' in stock_data and stock_data['total_mv'] is not None:
            try:
                total_mv = float(stock_data['total_mv'])
                if total_mv < 0:  # 总市值不应为负数
                    invalid_fields.append('total_mv')
            except (ValueError, TypeError):
                invalid_fields.append('total_mv')
                
        # 检查income_yoy是否为有效值
        if 'income_yoy' in stock_data and stock_data['income_yoy'] is not None:
            try:
                income_yoy = float(stock_data['income_yoy'])
                if income_yoy < -100 or income_yoy > 1000:  # 设置合理范围
                    invalid_fields.append('income_yoy')
            except (ValueError, TypeError):
                invalid_fields.append('income_yoy')
                
        # 验证结果
        is_valid = len(missing_fields) == 0 and len(invalid_fields) == 0
        
        result = {
            'valid': is_valid,
            'missing_fields': missing_fields,
            'invalid_fields': invalid_fields,
            'message': '数据有效' if is_valid else '数据存在问题'
        }
        
        return result
        
    def validate_historical_price_data(self, price_data):
        """
        验证历史价格数据的完整性和有效性
        
        Args:
            price_data: 历史价格数据字典
            
        Returns:
            dict: 验证结果
        """
        if not price_data:
            return {
                'valid': False,
                'missing_fields': ['all'],
                'invalid_fields': [],
                'message': '历史价格数据为空'
            }
            
        # 检查daily_prices是否存在
        if 'daily_prices' not in price_data or not price_data['daily_prices']:
            return {
                'valid': False,
                'missing_fields': ['daily_prices'],
                'invalid_fields': [],
                'message': '缺少daily_prices字段或为空'
            }
            
        # 必要字段列表
        required_fields = ['date', 'open', 'high', 'low', 'close', 'vol']
        
        # 检查样本数据点
        sample_size = min(10, len(price_data['daily_prices']))
        sample_data = price_data['daily_prices'][:sample_size]
        
        missing_fields = []
        invalid_fields = []
        
        for data_point in sample_data:
            # 检查缺失字段
            for field in required_fields:
                if field not in data_point or data_point[field] is None:
                    if field not in missing_fields:
                        missing_fields.append(field)
            
            # 检查价格字段是否为有效值
            for field in ['open', 'high', 'low', 'close']:
                if field in data_point and data_point[field] is not None:
                    try:
                        price = float(data_point[field])
                        if price < 0:  # 价格不应为负数
                            if field not in invalid_fields:
                                invalid_fields.append(field)
                    except (ValueError, TypeError):
                        if field not in invalid_fields:
                            invalid_fields.append(field)
                            
            # 检查成交量是否为有效值
            if 'vol' in data_point and data_point['vol'] is not None:
                try:
                    vol = float(data_point['vol'])
                    if vol < 0:  # 成交量不应为负数
                        if 'vol' not in invalid_fields:
                            invalid_fields.append('vol')
                except (ValueError, TypeError):
                    if 'vol' not in invalid_fields:
                        invalid_fields.append('vol')
        
        # 验证结果
        is_valid = len(missing_fields) == 0 and len(invalid_fields) == 0
        
        result = {
            'valid': is_valid,
            'missing_fields': missing_fields,
            'invalid_fields': invalid_fields,
            'message': '数据有效' if is_valid else '历史价格数据存在问题'
        }
        
        return result
        
    def validate_industry_data(self, industry_data):
        """
        验证行业数据的完整性和有效性
        
        Args:
            industry_data: 行业数据字典
            
        Returns:
            dict: 验证结果
        """
        if not industry_data:
            return {
                'valid': False,
                'message': '行业数据为空'
            }
            
        missing_industries = []
        empty_industries = []
        
        # 检查主要行业是否存在
        major_industries = [
            '银行', '证券', '保险', '房地产', '建筑', '建材', 
            '电力', '煤炭', '石油', '钢铁', '有色金属',
            '化工', '医药', '医疗器械', '生物技术', '电子', 
            '计算机', '传媒', '通信', '汽车', '机械', '家电',
            '食品饮料', '纺织服装', '商业贸易'
        ]
        
        for industry in major_industries:
            if industry not in industry_data:
                missing_industries.append(industry)
            elif not industry_data[industry]:
                empty_industries.append(industry)
                
        # 验证结果
        is_valid = len(missing_industries) == 0 and len(empty_industries) == 0
        
        result = {
            'valid': is_valid,
            'missing_industries': missing_industries,
            'empty_industries': empty_industries,
            'message': '数据有效' if is_valid else '行业数据存在问题'
        }
        
        return result
        
    def validate_all_stock_data(self, stock_data_dict):
        """
        验证所有股票数据
        
        Args:
            stock_data_dict: 股票数据字典，键为股票代码
            
        Returns:
            dict: 验证结果统计
        """
        if not stock_data_dict:
            return {
                'valid': False,
                'valid_count': 0,
                'invalid_count': 0,
                'total_count': 0,
                'message': '股票数据集为空'
            }
            
        valid_count = 0
        invalid_count = 0
        problem_stocks = {}
        
        # 验证每只股票的数据
        for stock_code, stock_data in stock_data_dict.items():
            result = self.validate_stock_basic_data(stock_data)
            
            if result['valid']:
                valid_count += 1
            else:
                invalid_count += 1
                problem_stocks[stock_code] = {
                    'name': stock_data.get('name', '未知'),
                    'issues': {
                        'missing_fields': result['missing_fields'],
                        'invalid_fields': result['invalid_fields']
                    }
                }
                
        # 验证结果
        total_count = valid_count + invalid_count
        valid_ratio = valid_count / total_count if total_count > 0 else 0
        
        result = {
            'valid': valid_ratio >= 0.9,  # 如果90%以上的股票数据有效，则认为整体有效
            'valid_count': valid_count,
            'invalid_count': invalid_count,
            'total_count': total_count,
            'valid_ratio': valid_ratio,
            'problem_stocks': problem_stocks,
            'message': f'股票数据验证完成，有效率: {valid_ratio:.2%}'
        }
        
        return result
        
    def validate_data_directory(self, directory):
        """
        验证数据目录中的文件
        
        Args:
            directory: 数据目录路径
            
        Returns:
            dict: 验证结果
        """
        if not os.path.exists(directory):
            return {
                'valid': False,
                'message': f'目录不存在: {directory}'
            }
            
        # 文件统计
        total_files = 0
        valid_files = 0
        invalid_files = 0
        
        # 问题文件列表
        problem_files = []
        
        # 遍历目录中的文件
        for root, _, files in os.walk(directory):
            for file in files:
                if file.endswith('.json'):
                    file_path = os.path.join(root, file)
                    total_files += 1
                    
                    try:
                        # 读取JSON文件
                        with open(file_path, 'r', encoding='utf-8') as f:
                            data = json.load(f)
                            
                        # 简单验证：检查是否为空或只有少量字段
                        if not data or (isinstance(data, dict) and len(data) < 3):
                            invalid_files += 1
                            problem_files.append({
                                'path': file_path,
                                'issue': '数据为空或缺少字段'
                            })
                        else:
                            valid_files += 1
                            
                    except Exception as e:
                        invalid_files += 1
                        problem_files.append({
                            'path': file_path,
                            'issue': f'读取或解析失败: {str(e)}'
                        })
                        
        # 验证结果
        valid_ratio = valid_files / total_files if total_files > 0 else 0
        
        result = {
            'valid': valid_ratio >= 0.95,  # 如果95%以上的文件有效，则认为目录有效
            'total_files': total_files,
            'valid_files': valid_files,
            'invalid_files': invalid_files,
            'valid_ratio': valid_ratio,
            'problem_files': problem_files,
            'message': f'目录验证完成，有效率: {valid_ratio:.2%}'
        }
        
        return result
        
    def generate_validation_report(self, output_path=None):
        """
        生成数据验证报告
        
        Args:
            output_path: 报告输出路径，默认为None（不保存文件）
            
        Returns:
            dict: 验证报告
        """
        # 汇总验证结果
        report = {
            'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            'summary': {
                'valid': all(result.get('valid', False) for result in self.validation_results.values()),
                'total_tests': len(self.validation_results),
                'passed_tests': sum(1 for result in self.validation_results.values() if result.get('valid', False)),
                'failed_tests': sum(1 for result in self.validation_results.values() if not result.get('valid', False))
            },
            'details': self.validation_results
        }
        
        # 保存报告
        if output_path:
            try:
                os.makedirs(os.path.dirname(output_path), exist_ok=True)
                with open(output_path, 'w', encoding='utf-8') as f:
                    json.dump(report, f, ensure_ascii=False, indent=2)
                logger.info(f"数据验证报告已保存至: {output_path}")
            except Exception as e:
                logger.error(f"保存数据验证报告失败: {e}")
                
        return report
        
    def run_validation_suite(self, data_root='data'):
        """
        运行完整的验证套件
        
        Args:
            data_root: 数据根目录
            
        Returns:
            dict: 验证报告
        """
        logger.info("开始运行数据验证套件")
        
        # 验证tushare_data目录
        tushare_dir = os.path.join(data_root, 'tushare_data')
        if os.path.exists(tushare_dir):
            self.validation_results['tushare_data'] = self.validate_data_directory(tushare_dir)
            logger.info(f"Tushare数据验证: {self.validation_results['tushare_data']['message']}")
            
        # 验证price_history目录
        price_dir = os.path.join(data_root, 'price_history')
        if os.path.exists(price_dir):
            self.validation_results['price_history'] = self.validate_data_directory(price_dir)
            logger.info(f"价格历史数据验证: {self.validation_results['price_history']['message']}")
            
        # 验证company_info目录
        company_dir = os.path.join(data_root, 'company_info')
        if os.path.exists(company_dir):
            self.validation_results['company_info'] = self.validate_data_directory(company_dir)
            logger.info(f"公司信息数据验证: {self.validation_results['company_info']['message']}")
            
        # 验证market_news目录
        news_dir = os.path.join(data_root, 'market_news')
        if os.path.exists(news_dir):
            self.validation_results['market_news'] = self.validate_data_directory(news_dir)
            logger.info(f"市场新闻数据验证: {self.validation_results['market_news']['message']}")
            
        # 生成并返回报告
        report_path = os.path.join(data_root, 'validation_report.json')
        return self.generate_validation_report(output_path=report_path)


# 用法示例
if __name__ == "__main__":
    validator = DataValidator()
    report = validator.run_validation_suite()
    print(f"数据验证完成，通过率: {report['summary']['passed_tests']/report['summary']['total_tests']:.2%}") 